import __init__
import sys
import os
import pandas as pd
import json
import urllib.request

import time
import random
import requests
from lxml import etree
import wencaicopy
"""
#todu
同花顺行情:http://q.10jqka.com.cn
20211125已经完成以下代码编写。
gn:同花顺概念
thshy:同花顺行业
"""
def ths_bk(ths_code="gn"):
    #>>>ths_bk(ths_code="gn")
    # !/usr/bin/env python3
    # -*- coding: utf-8 -*-
    """
    gn
    thshy :http://q.10jqka.com.cn/thshy/ #同花顺行业
    Created on Fri Nov 17 19:41:44 2017

    @author: Mr.ZeroW

    同花顺板块成分股
    """

    # 首先不同板块地址不同 http://q.10jqka.com.cn/gn/detail/order/desc/page/1/ajax/1/code/300018
    # 需要高出不同板块页数有多少，才能得出地址


    header = ths_header()
    if ths_code=="gn":
        a = requests.get("http://q.10jqka.com.cn/{}/".format(ths_code), headers=header)
    if ths_code=="thshy":
        a = requests.get("http://q.10jqka.com.cn/{}/".format(ths_code), headers=header)
    html = etree.HTML(a.text)

    gnbk = html.xpath('/html/body/div[2]/div[1]/div//div//div//a')
    thsgnbk = []
    for i in range(len(gnbk)):
        thsgnbk.append((gnbk[i].text))
    print(thsgnbk)
    # 板块代码
    bkcode = html.xpath('/html/body/div[2]/div[1]/div//div//div//a/@href')
    bkcode = list(map(lambda x: x.split('/')[-2], bkcode))
    data = {'Name': thsgnbk}

    # 存储
    gnbk = pd.DataFrame(data, index=bkcode)
    gnbk.to_csv('gnbk_{}.csv'.format(ths_code))

    print('板块名称以及代码已爬取，存储文件名：gnbk_{}.csv'.format(ths_code))
    # 导入板块名称和代码
    data = pd.read_csv('gnbk_{}.csv'.format(ths_code))
    # 建立数据框，四列【板块id, 板块name, 成分股id, 成分股name】
    start = time.time()
    temp = pd.DataFrame()
    for i in range(len(data)):
        bk_code = str(data.iloc[i, 0])
        name = str(data.iloc[i, 1])
        print(bk_code, name)
        if ths_code=="gn":
            _df = _md(bk_code, name)
        if ths_code=="thshy":
            _df = _md(bk_code, name,dtype="thshy")

        temp = pd.concat([temp, _df])
        time.sleep(random.randint(15, 20))
        if i%10==1:#临时保存
            temp.to_csv("筛选数据_{}.txt".format(ths_code), encoding="utf-8")
        if i%30==1:
            time.sleep(1800)

    end = time.time()
    print('爬取结束！！\n开始时间：%s\n结束时间：%s\n' % (time.ctime(end), time.ctime(start)))
    temp.to_csv("筛选数据_{}.txt".format(ths_code), encoding="utf-8")
def ths_header(v=""):
        v = wencaicopy.WencaiCookie.getHeXinVByHttp()
        header = {
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
            'Accept-Encoding': 'gzip, deflate',
            'Accept-Language': 'zh-CN,zh;q=0.9',
            'Cache-Control': 'no-cache',
            'Connection': 'keep-alive',
            'Cookie': 'Hm_lvt_78c58f01938e4d85eaf619eae71b4ed1=1631512319; v={}'.format(v),
            'Host': 'q.10jqka.com.cn',
            'Pragma': 'no-cache',
            'Upgrade-Insecure-Requests': '1',
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36  ',
        }
        return header
def _md(bk_code="307408",bk_name="",dtype="gn"):
    """
    
    :param bk_code: 
    :param bk_name: 
    :param dtype: ="gn"  "thshy"
    :return:
    #>>>_md()
    """
    #print("#")
    s_id = []
    s_name = []

    if dtype=="gn":
        url = 'http://q.10jqka.com.cn/gn/detail/code/' + bk_code + '/'
    if dtype=="thshy":
        url = 'http://q.10jqka.com.cn/thshy/detail/code/' + bk_code + '/'
    header = ths_header()
    a = requests.get(url, headers=header)
    # 得出板块成分股有多少页
    html = etree.HTML(a.text)
    result = html.xpath('//*[@id="m-page"]/span/text()')

    if len(result)==0:
        return None
    try:
        page = int(result[0].split('/')[-1])
        for j in range(page):
            page_n = str(j + 1)
            #             print(page_n)
            if dtype=="gn":
                curl = 'http://q.10jqka.com.cn/gn/detail/order/desc/page/' + page_n + '/ajax/1/code/' + bk_code
            if dtype=="thshy":
                curl ='http://q.10jqka.com.cn/thshy/detail/field/199112/order/desc/page/'+ page_n +'/ajax/1/code/'+bk_code
            if j % 3 == 1:#每3次更新header
                header = ths_header()
            a = requests.get(curl, headers=header)
            #->print(curl)
            html = etree.HTML(a.text)
            # 成分股代码
            stock_code = html.xpath('/html/body/table/tbody/tr/td[2]/a/text()')
            #             print(stock_code)
            # 成分股名称
            stock_name = html.xpath('/html/body/table/tbody/tr/td[3]/a/text()')
            #           print(stock_name)
            s_id.append(stock_code)
            s_name.append(stock_name)
        #将纬打击
        example = s_id
        example1 = [token for st in example for token in st]
        print(example1)
        example = s_name
        example2 = [token for st in example for token in st]
        print(example2)
        df_=pd.DataFrame({"S_ID":example1,"S_NAME":example2})
        df_["bk_id"]=bk_code
        df_["bk_name"] = bk_name
        print(df_)
        df_.to_csv("筛选数据_temp.txt", encoding="utf-8")
        return df_
    except:
        raise ("error")
if __name__ == '__main__':
    #_md()
    #_md(bk_code="881121",bk_name="",dtype="thshy")
    #ths_bk()
    ths_bk(ths_code="thshy")